# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from trispider.items import CourseItem

class CsdncourseSpider(scrapy.Spider):
    name = 'CSDNCourse'
    allowed_domains = ['edu.csdn.net']
    start_urls = ['http://edu.csdn.net/courses/k/p1']

    def parse(self, response):
        # item = CourseItem()
        # item["name"] = response.xpath("//div[@class='titleInfor']/span/text()").extract()
        # item["detailurl"] = response.xpath("//div[@class='course_dl_list']/a/@href").extract()
        # item["teacher"] = response.xpath("//div[@class='titleInfor']/following::p[1]/text()").extract()
        # item["price"] = response.xpath("//div[@class='titleInfor']/following::p[3]/i/text()").extract()
        # item["course_num"] = response.xpath("//div[@class='titleInfor']/following::p[2]/em[1]/text()").extract()
        print("crawl list page:%s" % response.url)
        detailurls = response.xpath("//div[@class='course_dl_list']/a/@href").extract()

        for i in range(0,len(detailurls)):
            url = detailurls[i]
            # print(url)
            yield Request(url,callback=self.getStudentNumber)

        nexturl = response.xpath("//span[@class='page-nav']/a[@class='btn btn-xs btn-default btn-next']/@href").extract()[0]

        if nexturl == response.url:
            return;
        yield Request(nexturl,callback=self.parse)

    def getStudentNumber(self, response):
        print("crawl detail page:%s" % response.url)

        item = CourseItem()
        item["name"] = response.xpath("//h1/a/text()").extract()[0]
        item["teacher"] = response.xpath("//div[@class='professor_name']/a/text()").extract()[0]
        item["price"] =  response.xpath("//span[@class='money']/text()").extract()[0]
        item["course_num"] = response.xpath("//span[@class='pinfo']/text()").extract()[0]
        item["student_num"] = response.xpath("//span[@class='howmany']/span[@class='num']/text()").extract()[0]
        yield item




